package com.example.demo;

import java.net.InetSocketAddress;
import java.net.Proxy;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;

import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import lombok.extern.slf4j.Slf4j;

/**
 * 使用指定cookie调用medium网站,返回内容.
 */
@Slf4j
@Component
public class MediumCrawler {
	@Value("${cookie}")
	String cookie;
	@Value("${baseDir}")
	String baseDir;
	@Value("${proxyHost}")
	String proxyHost;
	@Value("${proxyPort}")
	int proxyPort;
	
	@Autowired
	Translator translator;
	@Autowired
	PdfConverter pdfConverter;
	@Autowired
	ParseHeaderFromFile parseHeaderFromFile;

	RestTemplate restTemplate;
	String templateHtml;
	ExecutorService executorService;
	
	@PostConstruct
	public void init() {
		System.setProperty("http.proxyHost", proxyHost);
        System.setProperty("http.proxyPort", ""+proxyPort);
        System.setProperty("https.proxyHost", proxyHost);
        System.setProperty("https.proxyPort", ""+proxyPort);
        
        templateHtml = FileUtil.readString(baseDir+"\\template.html",StandardCharsets.UTF_8);
        executorService = Executors.newFixedThreadPool(10);
		
		SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory();
		InetSocketAddress address = new InetSocketAddress(proxyHost, proxyPort);
		Proxy proxy = new Proxy(Proxy.Type.HTTP, address);
		factory.setProxy(proxy);
		restTemplate = new RestTemplate(factory);
	}
	
	@PreDestroy
	public void stop() {
		executorService.shutdownNow();
	}

	public void download() {
		List<Article> articles = getTop10ByArticleSearch("software-engineering");
		log.info("----------top10 articles--------");
		articles.forEach(article -> System.out.println(article.getUniqueSlug()));
		downloadHtml(articles);
//		processArticle(articles.get(0)); //for test
		CountDownLatch countDownLatch = new CountDownLatch(articles.size());
		articles.forEach(article -> {
			executorService.submit(()->{
				try {
					processArticle(article);
				} catch (Exception e) {
					log.error("处理文章:{}失败:" + e.getMessage(), article.getUniqueSlug(), e);
				}finally {
					countDownLatch.countDown();
				}
			});
		});
		
		try {
			countDownLatch.await();
			log.info("所有文章处理完成.");
		} catch (InterruptedException e) {
			throw new RuntimeException(e);
		}
	}
	
	void downloadHtml(List<Article> articles) {
		articles.forEach(article -> {
			String htmlContent=request(HttpMethod.GET,"https://levelup.gitconnected.com/"+article.getUniqueSlug(), null, parseHeaderFromFile.getGitconnectedHeaders());
			article.html=htmlContent;
		});
	}
	
	void processArticle(Article article) {
		log.info("开始处理文章:{}",article.getUniqueSlug());
		article.setParagraphs(Util.parseParagraph(article.getHtml()));
		translateBatch(article);
		modifyHtmlByReplace(article);
		log.info("修改后的HTML: "+article.getHtml());
		String htmlPath = baseDir+"\\result\\html\\"+article.getId()+".html";
		String pdfPath = baseDir+"\\result\\pdf\\"+article.getId()+".pdf";
		FileUtil.writeString(article.getHtml(),htmlPath,StandardCharsets.UTF_8);
		pdfConverter.html2pdf(htmlPath, pdfPath);
		Util.sleepSeconds(3);
	}
	
	
	
	public static void main(String[] strs) {		
		MediumCrawler httpCaller = new MediumCrawler();
//		httpCaller.get(httpCaller.listArticleUri,null,ParseHeaderFromFile.parseHeaders());
//		httpCaller.queryClapCountByPostId("4f6d6305e133");
//		httpCaller.scanArticleList("software-engineering",null,0,5,5);
//		httpCaller.scanArticleList("software-engineering","345f0870-8815-4fea-b7b1-4cf9d66377c4",5,10,5);
//		List<Article> articles = httpCaller.scanArticleListByPage("software-engineering", 5, 2);
//		List<Article> articles = httpCaller.getTop10ByArticleSearch("software-engineering");
//		System.out.println("----------all articles--------");
//		articles.forEach(System.out::println);
//		System.out.println("----------topN--------");
//		articles=Util.getTopN(articles, 2);
//		articles.forEach(System.out::println);
//		
//		String htmlContent = FileUtil.readString("D:\\workspace\\demo\\awesome-terminal-applications-e4a06022dffa.html",
//				StandardCharsets.UTF_8);
//		String htmlContent= httpCaller.request(HttpMethod.GET,"https://levelup.gitconnected.com/awesome-terminal-applications-e4a06022dffa", null, parseHeaderFromFile.getGitconnectedHeaders());
//		List<Paragraph> ps = Util.parseParagraph(htmlContent);
////		ps=httpCaller.translateTextPGs(ps);
//		Article article = new Article();
//		article.setId("asdfdd");
//		article.setTitle("how-to-be-a-better-software-engineer ");
//		article.setParagraphs(ps);
//		String newHtml=httpCaller.createHtml(article);
//		System.out.println("newHtml:"+newHtml);
//		String dir = "D:\\workspace\\demo\\result\\";
//		String htmlPath = dir+article.getId()+".html";
//		String pdfPath = dir+article.getId()+".pdf";
//		FileUtil.writeString(newHtml,htmlPath,StandardCharsets.UTF_8);
//		httpCaller.pdfConverter.html2pdf(htmlPath, pdfPath);
	}
	
	// TODO 使用freemarker
	void modifyHtmlByTemplate(Article article) {
		String newHtml=templateHtml.replace("@title@", article.getTitle());
		StringBuilder sb=new StringBuilder();
		for(Paragraph p:article.getParagraphs()) {
			if("IMG".equalsIgnoreCase(p.getType())){
				sb.append("<IMG src=\"/"+p.getImgId()+"\">\n");
			}else {
				sb.append("<"+p.getType()+">"+p.getText()+"</"+p.getType()+">\n");
				//把翻译后的那段加上
				if(StrUtil.isNotBlank(p.getTranslatedText())) {
					sb.append("<"+p.getType()+">"+p.getTranslatedText()+"</"+p.getType()+">\n");
				}
			}			 
		}
		newHtml=newHtml.replace("@content@", sb.toString());
		article.html=newHtml;
	}
	
	/** 修改原html,替换原文为原文+译文,FIXME:相同内容可能被多次替换 */
	void modifyHtmlByReplace(Article article) {
		for(Paragraph p:article.getParagraphs()) {
			if(StrUtil.isNotBlank(p.getTranslatedText())) {	
				article.html=article.html.replace(p.getText(), p.getText()+"&nbsp;"+p.getTranslatedText());
			}
		}
		
		// 替换图片地址https://miro.medium.com/v2/resize:fit:720/format:webp/ 
		article.html=article.html.replaceAll("https://miro.medium.com/v2/resize:fit:(\\d+)/format:\\w+/", "");
	}

	@Deprecated
	List<Article> getTop10ByScan(String tag) {
		List<Article> articles = scanArticleListByPage("software-engineering", 5, 2);
		articles=Util.getTopN(articles, 10);
		return articles;
	}
	
	// 先查询OLDEST或NEWEST,再取顺带返回的mostReadFeed
	List<Article> getTop10ByArticleSearch(String tag) {
			String reqBody="[\r\n"
					+ "  {\r\n"
					+ "    \"operationName\": \"TagArchiveFeedQuery\",\r\n"
					+ "    \"variables\": {\r\n"
					+ "      \"tagSlug\": \""+tag+"\",\r\n"
					+ "      \"timeRange\": {\r\n"
					+ "        \"kind\": \"ALL_TIME\"\r\n"
					+ "      },\r\n"
					+ "      \"sortOrder\": \"OLDEST\",\r\n" // 或
					+ "      \"first\": 10  \r\n"
					+ "    },\r\n"
					+ "    \"query\": \"query TagArchiveFeedQuery($tagSlug: String!, $timeRange: TagPostsTimeRange!, $sortOrder: TagPostsSortOrder!, $first: Int!, $after: String) {\\n  tagFromSlug(tagSlug: $tagSlug) {\\n    id\\n    sortedFeed: posts(\\n      timeRange: $timeRange\\n      sortOrder: $sortOrder\\n      first: $first\\n      after: $after\\n    ) {\\n      ...TagPosts_tagPostConnection\\n      __typename\\n    }\\n    mostReadFeed: posts(timeRange: $timeRange, sortOrder: MOST_READ, first: $first) {\\n      ...TagPosts_tagPostConnection\\n      __typename\\n    }\\n    __typename\\n  }\\n}\\n\\nfragment TagPosts_tagPostConnection on TagPostConnection {\\n  edges {\\n    cursor\\n    node {\\n      id\\n      ...StreamPostPreview_post\\n      __typename\\n    }\\n    __typename\\n  }\\n  pageInfo {\\n    hasNextPage\\n    endCursor\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment StreamPostPreview_post on Post {\\n  id\\n  ...StreamPostPreviewContent_post\\n  ...PostPreviewContainer_post\\n  __typename\\n}\\n\\nfragment StreamPostPreviewContent_post on Post {\\n  id\\n  title\\n  previewImage {\\n    id\\n    __typename\\n  }\\n  extendedPreviewContent {\\n    subtitle\\n    __typename\\n  }\\n  ...StreamPostPreviewImage_post\\n  ...PostPreviewFooter_post\\n  ...PostPreviewByLine_post\\n  ...PostPreviewInformation_post\\n  __typename\\n}\\n\\nfragment StreamPostPreviewImage_post on Post {\\n  title\\n  previewImage {\\n    ...StreamPostPreviewImage_imageMetadata\\n    __typename\\n    id\\n  }\\n  __typename\\n  id\\n}\\n\\nfragment StreamPostPreviewImage_imageMetadata on ImageMetadata {\\n  id\\n  focusPercentX\\n  focusPercentY\\n  alt\\n  __typename\\n}\\n\\nfragment PostPreviewFooter_post on Post {\\n  ...PostPreviewFooterSocial_post\\n  ...PostPreviewFooterMenu_post\\n  __typename\\n  id\\n}\\n\\nfragment PostPreviewFooterSocial_post on Post {\\n  id\\n  ...MultiVote_post\\n  allowResponses\\n  isLimitedState\\n  postResponses {\\n    count\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment MultiVote_post on Post {\\n  id\\n  creator {\\n    id\\n    ...SusiClickable_user\\n    __typename\\n  }\\n  isPublished\\n  ...SusiClickable_post\\n  collection {\\n    id\\n    slug\\n    __typename\\n  }\\n  isLimitedState\\n  ...MultiVoteCount_post\\n  __typename\\n}\\n\\nfragment SusiClickable_user on User {\\n  ...SusiContainer_user\\n  __typename\\n  id\\n}\\n\\nfragment SusiContainer_user on User {\\n  ...SignInOptions_user\\n  ...SignUpOptions_user\\n  __typename\\n  id\\n}\\n\\nfragment SignInOptions_user on User {\\n  id\\n  name\\n  __typename\\n}\\n\\nfragment SignUpOptions_user on User {\\n  id\\n  name\\n  __typename\\n}\\n\\nfragment SusiClickable_post on Post {\\n  id\\n  mediumUrl\\n  ...SusiContainer_post\\n  __typename\\n}\\n\\nfragment SusiContainer_post on Post {\\n  id\\n  __typename\\n}\\n\\nfragment MultiVoteCount_post on Post {\\n  id\\n  __typename\\n}\\n\\nfragment PostPreviewFooterMenu_post on Post {\\n  creator {\\n    __typename\\n    id\\n  }\\n  collection {\\n    __typename\\n    id\\n  }\\n  ...BookmarkButton_post\\n  ...ExpandablePostCardOverflowButton_post\\n  __typename\\n  id\\n}\\n\\nfragment BookmarkButton_post on Post {\\n  visibility\\n  ...SusiClickable_post\\n  ...AddToCatalogBookmarkButton_post\\n  __typename\\n  id\\n}\\n\\nfragment AddToCatalogBookmarkButton_post on Post {\\n  ...AddToCatalogBase_post\\n  __typename\\n  id\\n}\\n\\nfragment AddToCatalogBase_post on Post {\\n  id\\n  isPublished\\n  __typename\\n}\\n\\nfragment ExpandablePostCardOverflowButton_post on Post {\\n  creator {\\n    id\\n    __typename\\n  }\\n  ...ExpandablePostCardReaderButton_post\\n  __typename\\n  id\\n}\\n\\nfragment ExpandablePostCardReaderButton_post on Post {\\n  id\\n  collection {\\n    id\\n    __typename\\n  }\\n  creator {\\n    id\\n    __typename\\n  }\\n  clapCount\\n  ...ClapMutation_post\\n  __typename\\n}\\n\\nfragment ClapMutation_post on Post {\\n  __typename\\n  id\\n  clapCount\\n  ...MultiVoteCount_post\\n}\\n\\nfragment PostPreviewByLine_post on Post {\\n  id\\n  creator {\\n    ...PostPreviewByLine_user\\n    __typename\\n    id\\n  }\\n  collection {\\n    ...PostPreviewByLine_collection\\n    __typename\\n    id\\n  }\\n  ...CardByline_post\\n  __typename\\n}\\n\\nfragment PostPreviewByLine_user on User {\\n  id\\n  __typename\\n  ...CardByline_user\\n  ...ExpandablePostByline_user\\n}\\n\\nfragment CardByline_user on User {\\n  __typename\\n  id\\n  name\\n  username\\n  mediumMemberAt\\n  socialStats {\\n    followerCount\\n    __typename\\n  }\\n  ...useIsVerifiedBookAuthor_user\\n  ...userUrl_user\\n  ...UserMentionTooltip_user\\n}\\n\\nfragment useIsVerifiedBookAuthor_user on User {\\n  verifications {\\n    isBookAuthor\\n    __typename\\n  }\\n  __typename\\n  id\\n}\\n\\nfragment userUrl_user on User {\\n  __typename\\n  id\\n  customDomainState {\\n    live {\\n      domain\\n      __typename\\n    }\\n    __typename\\n  }\\n  hasSubdomain\\n  username\\n}\\n\\nfragment UserMentionTooltip_user on User {\\n  id\\n  name\\n  username\\n  bio\\n  imageId\\n  mediumMemberAt\\n  membership {\\n    tier\\n    __typename\\n    id\\n  }\\n  ...UserAvatar_user\\n  ...UserFollowButton_user\\n  ...useIsVerifiedBookAuthor_user\\n  __typename\\n}\\n\\nfragment UserAvatar_user on User {\\n  __typename\\n  id\\n  imageId\\n  mediumMemberAt\\n  membership {\\n    tier\\n    __typename\\n    id\\n  }\\n  name\\n  username\\n  ...userUrl_user\\n}\\n\\nfragment UserFollowButton_user on User {\\n  ...UserFollowButtonSignedIn_user\\n  ...UserFollowButtonSignedOut_user\\n  __typename\\n  id\\n}\\n\\nfragment UserFollowButtonSignedIn_user on User {\\n  id\\n  name\\n  __typename\\n}\\n\\nfragment UserFollowButtonSignedOut_user on User {\\n  id\\n  ...SusiClickable_user\\n  __typename\\n}\\n\\nfragment ExpandablePostByline_user on User {\\n  __typename\\n  id\\n  name\\n  imageId\\n  ...userUrl_user\\n  ...useIsVerifiedBookAuthor_user\\n}\\n\\nfragment PostPreviewByLine_collection on Collection {\\n  id\\n  __typename\\n  ...CardByline_collection\\n  ...CollectionLinkWithPopover_collection\\n}\\n\\nfragment CardByline_collection on Collection {\\n  name\\n  ...collectionUrl_collection\\n  __typename\\n  id\\n}\\n\\nfragment collectionUrl_collection on Collection {\\n  id\\n  domain\\n  slug\\n  __typename\\n}\\n\\nfragment CollectionLinkWithPopover_collection on Collection {\\n  ...collectionUrl_collection\\n  ...CollectionTooltip_collection\\n  __typename\\n  id\\n}\\n\\nfragment CollectionTooltip_collection on Collection {\\n  id\\n  name\\n  description\\n  subscriberCount\\n  ...CollectionAvatar_collection\\n  ...CollectionFollowButton_collection\\n  __typename\\n}\\n\\nfragment CollectionAvatar_collection on Collection {\\n  name\\n  avatar {\\n    id\\n    __typename\\n  }\\n  ...collectionUrl_collection\\n  __typename\\n  id\\n}\\n\\nfragment CollectionFollowButton_collection on Collection {\\n  __typename\\n  id\\n  name\\n  slug\\n  ...collectionUrl_collection\\n  ...SusiClickable_collection\\n}\\n\\nfragment SusiClickable_collection on Collection {\\n  ...SusiContainer_collection\\n  __typename\\n  id\\n}\\n\\nfragment SusiContainer_collection on Collection {\\n  name\\n  ...SignInOptions_collection\\n  ...SignUpOptions_collection\\n  __typename\\n  id\\n}\\n\\nfragment SignInOptions_collection on Collection {\\n  id\\n  name\\n  __typename\\n}\\n\\nfragment SignUpOptions_collection on Collection {\\n  id\\n  name\\n  __typename\\n}\\n\\nfragment CardByline_post on Post {\\n  ...DraftStatus_post\\n  ...Star_post\\n  ...shouldShowPublishedInStatus_post\\n  __typename\\n  id\\n}\\n\\nfragment DraftStatus_post on Post {\\n  id\\n  pendingCollection {\\n    id\\n    creator {\\n      id\\n      __typename\\n    }\\n    ...BoldCollectionName_collection\\n    __typename\\n  }\\n  statusForCollection\\n  creator {\\n    id\\n    __typename\\n  }\\n  isPublished\\n  __typename\\n}\\n\\nfragment BoldCollectionName_collection on Collection {\\n  id\\n  name\\n  __typename\\n}\\n\\nfragment Star_post on Post {\\n  id\\n  creator {\\n    id\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment shouldShowPublishedInStatus_post on Post {\\n  statusForCollection\\n  isPublished\\n  __typename\\n  id\\n}\\n\\nfragment PostPreviewInformation_post on Post {\\n  pinnedAt\\n  latestPublishedAt\\n  firstPublishedAt\\n  readingTime\\n  isLocked\\n  ...Star_post\\n  __typename\\n  id\\n}\\n\\nfragment PostPreviewContainer_post on Post {\\n  id\\n  extendedPreviewContent {\\n    isFullContent\\n    __typename\\n  }\\n  visibility\\n  pinnedAt\\n  ...PostScrollTracker_post\\n  ...usePostUrl_post\\n  __typename\\n}\\n\\nfragment PostScrollTracker_post on Post {\\n  id\\n  collection {\\n    id\\n    __typename\\n  }\\n  sequence {\\n    sequenceId\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment usePostUrl_post on Post {\\n  id\\n  creator {\\n    ...userUrl_user\\n    __typename\\n    id\\n  }\\n  collection {\\n    id\\n    domain\\n    slug\\n    __typename\\n  }\\n  isSeries\\n  mediumUrl\\n  sequence {\\n    slug\\n    __typename\\n  }\\n  uniqueSlug\\n  __typename\\n}\\n\"\r\n"
					+ "  }\r\n"
					+ "]";
			
			Map<String, String> headers = parseHeaderFromFile.getGraphQLHeaders();
			headers.put("Graphql-Operation", "TagArchiveFeedQuery");
			headers.put("Medium-Frontend-Path", "/?tag=" + tag);
			headers.put("Medium-Frontend-Route", "homepage");
			String resBody = request(HttpMethod.POST,"https://medium.com/_/graphql", reqBody, headers);

			JSONArray arr = JSONUtil.parseArray(resBody);
			JSONObject obj = (JSONObject) arr.get(0);

			List<Article> articles = new ArrayList<>();
			@SuppressWarnings("unchecked")
			List<JSONObject> posts = (List<JSONObject>) JSONUtil.getByPath(obj, "data.tagFromSlug.mostReadFeed.edges.node");
			posts.forEach(post -> {
				Article article = new Article();
				article.setId((String) post.get("id"));
				article.setClapCount((int) post.get("clapCount"));
				article.setTitle((String) post.get("title"));
				article.setUniqueSlug((String) post.get("uniqueSlug"));
				log.info("article:{}", article);
				articles.add(article);
			});		
			
			return articles;
	}
	
	/**
	 * 将文本类型的段落翻译为中文,插入到原始段落的后面
	 * TODO: 将所有段落一起用分割符拼接,发给百度翻译,然后分拆
	 */
//	void translate(Article article) {
//		List<Paragraph> origPgs = article.getParagraphs();
//		origPgs.forEach(paragraph -> {
//			String text = paragraph.getText();
//			if (StrUtil.isBlank(text)) {
//				return;
//			}
//			Paragraph pg = ObjectUtil.clone(paragraph);
//			String text2 = translator.translateByOpenAI(text);
//			log.info("translate [{}] to [{}]",text,text2);
//			pg.setTranslatedText(text2);
//		});
//	}
	
	void translateBatch(Article article) {
		List<Paragraph> origPgs = article.getParagraphs();
		StringBuilder sb=new StringBuilder();
		int i=0;
		Map<Integer,Paragraph> map=new LinkedHashMap<>();
		for(Paragraph paragraph:origPgs){
			if(StrUtil.isBlank(paragraph.getText())){
				continue;
			}
			i++;
			String tag="@"+i+"@";
			sb.append(tag).append(StrUtil.trimToEmpty(paragraph.getText())).append(tag);
			map.put(i, paragraph);
		}
		
		String transResullt=translator.translateByOpenAI(sb.toString());
		for(int m=1;m<=3;m++) {
			if(!transResullt.contains("@1@")) {
				log.error("译文中丢失了段落序号标记,将第{}次重新翻译!!!",m);
				transResullt=translator.translateByOpenAI(sb.toString());
			}
		}
		
		if(!transResullt.contains("@1@")) {
			log.error("译文中丢失了段落序号标记翻译彻底失败!!");
			return;
		}
		
		for(int k=1;k<=i;k++) {
        	String tag="@"+k+"@";
        	int index=transResullt.indexOf(tag);
        	int index2=transResullt.indexOf(tag,index+tag.length());
        	if(index<0||index2<0||index>=index2) {
	        	continue;
        	}
        	String chinese=transResullt.substring(transResullt.indexOf(tag)+tag.length(), transResullt.lastIndexOf(tag));
        	Paragraph paragraph = map.get(k);
        	paragraph.setTranslatedText(chinese);
        	log.info("翻译段落:{} 为: {}",paragraph.getText(),chinese);
        }
		
	}

	// 分多也查询,limit=pageSize
	@Deprecated
	List<Article> scanArticleListByPage(String tag, int count, int limit) {
		List<Article> articles = new ArrayList<>();
		String source = null;
		for (int from = 0; from < count; from += limit) {
			limit = Math.min(count - from, limit);
			source = scanArticleList(tag, source, from, from + limit, limit, articles);
			if (StrUtil.isBlank(source)) {
				log.info("没有下一页或出问题了!");
				break;
			}
		}
		try {
			Thread.sleep(5000L);
		} catch (InterruptedException e) {
			throw new RuntimeException(e.getMessage(), e);
		}
		log.info("结果文章数={}", articles.size());
		return articles;
	}

	// 查询一页
	@Deprecated
	String scanArticleList(String tag, String source, int from, int to, int limit, List<Article> articles) {
		String reqBody = "[\r\n" + "  {\r\n" + "    \"operationName\": \"WebInlineTopicFeedQuery\",\r\n"
				+ "    \"variables\": {\r\n" + "      \"tagSlug\": \"" + tag + "\",\r\n" + "      \"paging\": {\r\n"
				+ (source == null ? "" : "        \"source\": \"" + source + "\",\r\n") + "        \"from\": \"" + from
				+ "\",\r\n" + "        \"to\": \"" + to + "\",\r\n" + "        \"limit\": " + limit + "\r\n"
				+ "      },\r\n" + "      \"skipCache\": true\r\n" + "    },\r\n"
				+ "    \"query\": \"query WebInlineTopicFeedQuery($tagSlug: String!, $paging: PagingOptions!, $skipCache: Boolean) {\\n  personalisedTagFeed(tagSlug: $tagSlug, paging: $paging, skipCache: $skipCache) {\\n    items {\\n      ... on TagFeedItem {\\n        feedId\\n        reason\\n        moduleSourceEncoding\\n        post {\\n          ...PostPreview_post\\n          __typename\\n        }\\n        __typename\\n      }\\n      __typename\\n    }\\n    pagingInfo {\\n      next {\\n        source\\n        limit\\n        from\\n        to\\n        __typename\\n      }\\n      __typename\\n    }\\n    __typename\\n  }\\n}\\n\\nfragment PostPreview_post on Post {\\n  id\\n  creator {\\n    ...PostPreview_user\\n    __typename\\n    id\\n  }\\n  collection {\\n    ...CardByline_collection\\n    ...ExpandablePostByline_collection\\n    __typename\\n    id\\n  }\\n  ...InteractivePostBody_postPreview\\n  firstPublishedAt\\n  isLocked\\n  isSeries\\n  latestPublishedAt\\n  inResponseToCatalogResult {\\n    __typename\\n  }\\n  pinnedAt\\n  pinnedByCreatorAt\\n  previewImage {\\n    id\\n    focusPercentX\\n    focusPercentY\\n    __typename\\n  }\\n  readingTime\\n  sequence {\\n    slug\\n    __typename\\n  }\\n  title\\n  uniqueSlug\\n  ...CardByline_post\\n  ...PostFooterActionsBar_post\\n  ...InResponseToEntityPreview_post\\n  ...PostScrollTracker_post\\n  ...HighDensityPreview_post\\n  __typename\\n}\\n\\nfragment PostPreview_user on User {\\n  __typename\\n  name\\n  username\\n  ...CardByline_user\\n  ...ExpandablePostByline_user\\n  id\\n}\\n\\nfragment CardByline_user on User {\\n  __typename\\n  id\\n  name\\n  username\\n  mediumMemberAt\\n  socialStats {\\n    followerCount\\n    __typename\\n  }\\n  ...useIsVerifiedBookAuthor_user\\n  ...userUrl_user\\n  ...UserMentionTooltip_user\\n}\\n\\nfragment useIsVerifiedBookAuthor_user on User {\\n  verifications {\\n    isBookAuthor\\n    __typename\\n  }\\n  __typename\\n  id\\n}\\n\\nfragment userUrl_user on User {\\n  __typename\\n  id\\n  customDomainState {\\n    live {\\n      domain\\n      __typename\\n    }\\n    __typename\\n  }\\n  hasSubdomain\\n  username\\n}\\n\\nfragment UserMentionTooltip_user on User {\\n  id\\n  name\\n  username\\n  bio\\n  imageId\\n  mediumMemberAt\\n  membership {\\n    tier\\n    __typename\\n    id\\n  }\\n  ...UserAvatar_user\\n  ...UserFollowButton_user\\n  ...useIsVerifiedBookAuthor_user\\n  __typename\\n}\\n\\nfragment UserAvatar_user on User {\\n  __typename\\n  id\\n  imageId\\n  mediumMemberAt\\n  membership {\\n    tier\\n    __typename\\n    id\\n  }\\n  name\\n  username\\n  ...userUrl_user\\n}\\n\\nfragment UserFollowButton_user on User {\\n  ...UserFollowButtonSignedIn_user\\n  ...UserFollowButtonSignedOut_user\\n  __typename\\n  id\\n}\\n\\nfragment UserFollowButtonSignedIn_user on User {\\n  id\\n  name\\n  __typename\\n}\\n\\nfragment UserFollowButtonSignedOut_user on User {\\n  id\\n  ...SusiClickable_user\\n  __typename\\n}\\n\\nfragment SusiClickable_user on User {\\n  ...SusiContainer_user\\n  __typename\\n  id\\n}\\n\\nfragment SusiContainer_user on User {\\n  ...SignInOptions_user\\n  ...SignUpOptions_user\\n  __typename\\n  id\\n}\\n\\nfragment SignInOptions_user on User {\\n  id\\n  name\\n  __typename\\n}\\n\\nfragment SignUpOptions_user on User {\\n  id\\n  name\\n  __typename\\n}\\n\\nfragment ExpandablePostByline_user on User {\\n  __typename\\n  id\\n  name\\n  imageId\\n  ...userUrl_user\\n  ...useIsVerifiedBookAuthor_user\\n}\\n\\nfragment CardByline_collection on Collection {\\n  name\\n  ...collectionUrl_collection\\n  __typename\\n  id\\n}\\n\\nfragment collectionUrl_collection on Collection {\\n  id\\n  domain\\n  slug\\n  __typename\\n}\\n\\nfragment ExpandablePostByline_collection on Collection {\\n  __typename\\n  id\\n  name\\n  domain\\n  slug\\n}\\n\\nfragment InteractivePostBody_postPreview on Post {\\n  extendedPreviewContent(\\n    truncationConfig: {previewParagraphsWordCountThreshold: 400, minimumWordLengthForTruncation: 150, truncateAtEndOfSentence: true, showFullImageCaptions: true, shortformPreviewParagraphsWordCountThreshold: 30, shortformMinimumWordLengthForTruncation: 30}\\n  ) {\\n    bodyModel {\\n      ...PostBody_bodyModel\\n      __typename\\n    }\\n    isFullContent\\n    __typename\\n  }\\n  __typename\\n  id\\n}\\n\\nfragment PostBody_bodyModel on RichText {\\n  sections {\\n    name\\n    startIndex\\n    textLayout\\n    imageLayout\\n    backgroundImage {\\n      id\\n      originalHeight\\n      originalWidth\\n      __typename\\n    }\\n    videoLayout\\n    backgroundVideo {\\n      videoId\\n      originalHeight\\n      originalWidth\\n      previewImageId\\n      __typename\\n    }\\n    __typename\\n  }\\n  paragraphs {\\n    id\\n    ...PostBodySection_paragraph\\n    __typename\\n  }\\n  ...normalizedBodyModel_richText\\n  __typename\\n}\\n\\nfragment PostBodySection_paragraph on Paragraph {\\n  name\\n  ...PostBodyParagraph_paragraph\\n  __typename\\n  id\\n}\\n\\nfragment PostBodyParagraph_paragraph on Paragraph {\\n  name\\n  type\\n  ...ImageParagraph_paragraph\\n  ...TextParagraph_paragraph\\n  ...IframeParagraph_paragraph\\n  ...MixtapeParagraph_paragraph\\n  ...CodeBlockParagraph_paragraph\\n  __typename\\n  id\\n}\\n\\nfragment ImageParagraph_paragraph on Paragraph {\\n  href\\n  layout\\n  metadata {\\n    id\\n    originalHeight\\n    originalWidth\\n    focusPercentX\\n    focusPercentY\\n    alt\\n    __typename\\n  }\\n  ...Markups_paragraph\\n  ...ParagraphRefsMapContext_paragraph\\n  ...PostAnnotationsMarker_paragraph\\n  __typename\\n  id\\n}\\n\\nfragment Markups_paragraph on Paragraph {\\n  name\\n  text\\n  hasDropCap\\n  dropCapImage {\\n    ...MarkupNode_data_dropCapImage\\n    __typename\\n    id\\n  }\\n  markups {\\n    ...Markups_markup\\n    __typename\\n  }\\n  __typename\\n  id\\n}\\n\\nfragment MarkupNode_data_dropCapImage on ImageMetadata {\\n  ...DropCap_image\\n  __typename\\n  id\\n}\\n\\nfragment DropCap_image on ImageMetadata {\\n  id\\n  originalHeight\\n  originalWidth\\n  __typename\\n}\\n\\nfragment Markups_markup on Markup {\\n  type\\n  start\\n  end\\n  href\\n  anchorType\\n  userId\\n  linkMetadata {\\n    httpStatus\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment ParagraphRefsMapContext_paragraph on Paragraph {\\n  id\\n  name\\n  text\\n  __typename\\n}\\n\\nfragment PostAnnotationsMarker_paragraph on Paragraph {\\n  ...PostViewNoteCard_paragraph\\n  __typename\\n  id\\n}\\n\\nfragment PostViewNoteCard_paragraph on Paragraph {\\n  name\\n  __typename\\n  id\\n}\\n\\nfragment TextParagraph_paragraph on Paragraph {\\n  type\\n  hasDropCap\\n  codeBlockMetadata {\\n    mode\\n    lang\\n    __typename\\n  }\\n  ...Markups_paragraph\\n  ...ParagraphRefsMapContext_paragraph\\n  __typename\\n  id\\n}\\n\\nfragment IframeParagraph_paragraph on Paragraph {\\n  type\\n  iframe {\\n    mediaResource {\\n      id\\n      iframeSrc\\n      iframeHeight\\n      iframeWidth\\n      title\\n      __typename\\n    }\\n    __typename\\n  }\\n  layout\\n  ...Markups_paragraph\\n  __typename\\n  id\\n}\\n\\nfragment MixtapeParagraph_paragraph on Paragraph {\\n  type\\n  mixtapeMetadata {\\n    href\\n    mediaResource {\\n      mediumCatalog {\\n        id\\n        __typename\\n      }\\n      __typename\\n    }\\n    __typename\\n  }\\n  ...GenericMixtapeParagraph_paragraph\\n  __typename\\n  id\\n}\\n\\nfragment GenericMixtapeParagraph_paragraph on Paragraph {\\n  text\\n  mixtapeMetadata {\\n    href\\n    thumbnailImageId\\n    __typename\\n  }\\n  markups {\\n    start\\n    end\\n    type\\n    href\\n    __typename\\n  }\\n  __typename\\n  id\\n}\\n\\nfragment CodeBlockParagraph_paragraph on Paragraph {\\n  codeBlockMetadata {\\n    lang\\n    mode\\n    __typename\\n  }\\n  __typename\\n  id\\n}\\n\\nfragment normalizedBodyModel_richText on RichText {\\n  paragraphs {\\n    ...normalizedBodyModel_richText_paragraphs\\n    __typename\\n  }\\n  sections {\\n    startIndex\\n    ...getSectionEndIndex_section\\n    __typename\\n  }\\n  ...getParagraphStyles_richText\\n  ...getParagraphSpaces_richText\\n  __typename\\n}\\n\\nfragment normalizedBodyModel_richText_paragraphs on Paragraph {\\n  markups {\\n    ...normalizedBodyModel_richText_paragraphs_markups\\n    __typename\\n  }\\n  codeBlockMetadata {\\n    lang\\n    mode\\n    __typename\\n  }\\n  ...getParagraphHighlights_paragraph\\n  ...getParagraphPrivateNotes_paragraph\\n  __typename\\n  id\\n}\\n\\nfragment normalizedBodyModel_richText_paragraphs_markups on Markup {\\n  type\\n  __typename\\n}\\n\\nfragment getParagraphHighlights_paragraph on Paragraph {\\n  name\\n  __typename\\n  id\\n}\\n\\nfragment getParagraphPrivateNotes_paragraph on Paragraph {\\n  name\\n  __typename\\n  id\\n}\\n\\nfragment getSectionEndIndex_section on Section {\\n  startIndex\\n  __typename\\n}\\n\\nfragment getParagraphStyles_richText on RichText {\\n  paragraphs {\\n    text\\n    type\\n    __typename\\n  }\\n  sections {\\n    ...getSectionEndIndex_section\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment getParagraphSpaces_richText on RichText {\\n  paragraphs {\\n    layout\\n    metadata {\\n      originalHeight\\n      originalWidth\\n      id\\n      __typename\\n    }\\n    type\\n    ...paragraphExtendsImageGrid_paragraph\\n    __typename\\n  }\\n  ...getSeriesParagraphTopSpacings_richText\\n  ...getPostParagraphTopSpacings_richText\\n  __typename\\n}\\n\\nfragment paragraphExtendsImageGrid_paragraph on Paragraph {\\n  layout\\n  type\\n  __typename\\n  id\\n}\\n\\nfragment getSeriesParagraphTopSpacings_richText on RichText {\\n  paragraphs {\\n    id\\n    __typename\\n  }\\n  sections {\\n    ...getSectionEndIndex_section\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment getPostParagraphTopSpacings_richText on RichText {\\n  paragraphs {\\n    type\\n    layout\\n    text\\n    codeBlockMetadata {\\n      lang\\n      mode\\n      __typename\\n    }\\n    __typename\\n  }\\n  sections {\\n    ...getSectionEndIndex_section\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment CardByline_post on Post {\\n  ...DraftStatus_post\\n  ...Star_post\\n  ...shouldShowPublishedInStatus_post\\n  __typename\\n  id\\n}\\n\\nfragment DraftStatus_post on Post {\\n  id\\n  pendingCollection {\\n    id\\n    creator {\\n      id\\n      __typename\\n    }\\n    ...BoldCollectionName_collection\\n    __typename\\n  }\\n  statusForCollection\\n  creator {\\n    id\\n    __typename\\n  }\\n  isPublished\\n  __typename\\n}\\n\\nfragment BoldCollectionName_collection on Collection {\\n  id\\n  name\\n  __typename\\n}\\n\\nfragment Star_post on Post {\\n  id\\n  creator {\\n    id\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment shouldShowPublishedInStatus_post on Post {\\n  statusForCollection\\n  isPublished\\n  __typename\\n  id\\n}\\n\\nfragment PostFooterActionsBar_post on Post {\\n  id\\n  visibility\\n  allowResponses\\n  postResponses {\\n    count\\n    __typename\\n  }\\n  isLimitedState\\n  creator {\\n    id\\n    __typename\\n  }\\n  collection {\\n    id\\n    __typename\\n  }\\n  ...MultiVote_post\\n  ...PostSharePopover_post\\n  ...OverflowMenuButtonWithNegativeSignal_post\\n  ...PostPageBookmarkButton_post\\n  __typename\\n}\\n\\nfragment MultiVote_post on Post {\\n  id\\n  creator {\\n    id\\n    ...SusiClickable_user\\n    __typename\\n  }\\n  isPublished\\n  ...SusiClickable_post\\n  collection {\\n    id\\n    slug\\n    __typename\\n  }\\n  isLimitedState\\n  ...MultiVoteCount_post\\n  __typename\\n}\\n\\nfragment SusiClickable_post on Post {\\n  id\\n  mediumUrl\\n  ...SusiContainer_post\\n  __typename\\n}\\n\\nfragment SusiContainer_post on Post {\\n  id\\n  __typename\\n}\\n\\nfragment MultiVoteCount_post on Post {\\n  id\\n  __typename\\n}\\n\\nfragment PostSharePopover_post on Post {\\n  id\\n  mediumUrl\\n  title\\n  isPublished\\n  isLocked\\n  ...usePostUrl_post\\n  ...FriendLink_post\\n  __typename\\n}\\n\\nfragment usePostUrl_post on Post {\\n  id\\n  creator {\\n    ...userUrl_user\\n    __typename\\n    id\\n  }\\n  collection {\\n    id\\n    domain\\n    slug\\n    __typename\\n  }\\n  isSeries\\n  mediumUrl\\n  sequence {\\n    slug\\n    __typename\\n  }\\n  uniqueSlug\\n  __typename\\n}\\n\\nfragment FriendLink_post on Post {\\n  id\\n  ...SusiClickable_post\\n  ...useCopyFriendLink_post\\n  ...UpsellClickable_post\\n  __typename\\n}\\n\\nfragment useCopyFriendLink_post on Post {\\n  ...usePostUrl_post\\n  __typename\\n  id\\n}\\n\\nfragment UpsellClickable_post on Post {\\n  id\\n  collection {\\n    id\\n    __typename\\n  }\\n  sequence {\\n    sequenceId\\n    __typename\\n  }\\n  creator {\\n    id\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment OverflowMenuButtonWithNegativeSignal_post on Post {\\n  id\\n  visibility\\n  ...OverflowMenuWithNegativeSignal_post\\n  __typename\\n}\\n\\nfragment OverflowMenuWithNegativeSignal_post on Post {\\n  id\\n  creator {\\n    id\\n    __typename\\n  }\\n  collection {\\n    id\\n    __typename\\n  }\\n  ...OverflowMenuItemUndoClaps_post\\n  ...AddToCatalogBase_post\\n  __typename\\n}\\n\\nfragment OverflowMenuItemUndoClaps_post on Post {\\n  id\\n  clapCount\\n  ...ClapMutation_post\\n  __typename\\n}\\n\\nfragment ClapMutation_post on Post {\\n  __typename\\n  id\\n  clapCount\\n  ...MultiVoteCount_post\\n}\\n\\nfragment AddToCatalogBase_post on Post {\\n  id\\n  isPublished\\n  __typename\\n}\\n\\nfragment PostPageBookmarkButton_post on Post {\\n  ...AddToCatalogBookmarkButton_post\\n  __typename\\n  id\\n}\\n\\nfragment AddToCatalogBookmarkButton_post on Post {\\n  ...AddToCatalogBase_post\\n  __typename\\n  id\\n}\\n\\nfragment InResponseToEntityPreview_post on Post {\\n  id\\n  inResponseToEntityType\\n  __typename\\n}\\n\\nfragment PostScrollTracker_post on Post {\\n  id\\n  collection {\\n    id\\n    __typename\\n  }\\n  sequence {\\n    sequenceId\\n    __typename\\n  }\\n  __typename\\n}\\n\\nfragment HighDensityPreview_post on Post {\\n  id\\n  title\\n  previewImage {\\n    id\\n    focusPercentX\\n    focusPercentY\\n    __typename\\n  }\\n  extendedPreviewContent(\\n    truncationConfig: {previewParagraphsWordCountThreshold: 400, minimumWordLengthForTruncation: 150, truncateAtEndOfSentence: true, showFullImageCaptions: true, shortformPreviewParagraphsWordCountThreshold: 30, shortformMinimumWordLengthForTruncation: 30}\\n  ) {\\n    subtitle\\n    __typename\\n  }\\n  ...HighDensityFooter_post\\n  __typename\\n}\\n\\nfragment HighDensityFooter_post on Post {\\n  id\\n  readingTime\\n  tags {\\n    ...TopicPill_tag\\n    __typename\\n  }\\n  ...BookmarkButton_post\\n  ...ExpandablePostCardOverflowButton_post\\n  ...OverflowMenuButtonWithNegativeSignal_post\\n  __typename\\n}\\n\\nfragment TopicPill_tag on Tag {\\n  __typename\\n  id\\n  displayTitle\\n  normalizedTagSlug\\n}\\n\\nfragment BookmarkButton_post on Post {\\n  visibility\\n  ...SusiClickable_post\\n  ...AddToCatalogBookmarkButton_post\\n  __typename\\n  id\\n}\\n\\nfragment ExpandablePostCardOverflowButton_post on Post {\\n  creator {\\n    id\\n    __typename\\n  }\\n  ...ExpandablePostCardReaderButton_post\\n  __typename\\n  id\\n}\\n\\nfragment ExpandablePostCardReaderButton_post on Post {\\n  id\\n  collection {\\n    id\\n    __typename\\n  }\\n  creator {\\n    id\\n    __typename\\n  }\\n  clapCount\\n  ...ClapMutation_post\\n  __typename\\n}\\n\""
				+ "}]";
		Map<String, String> headers = parseHeaderFromFile.parseGraphQLHeaders();
		headers.put("Graphql-Operation", "WebInlineTopicFeedQuery");
		headers.put("Medium-Frontend-Path", "/?tag=" + tag);
		headers.put("Graphql-Operation", "WebInlineTopicFeedQuery");
		headers.put("Medium-Frontend-Route", "homepage");
		String resBody = request(HttpMethod.POST,"https://medium.com/_/graphql", reqBody, headers);

		JSONArray arr = JSONUtil.parseArray(resBody);
		JSONObject obj = (JSONObject) arr.get(0);

		@SuppressWarnings("unchecked")
		List<JSONObject> posts = (List<JSONObject>) JSONUtil.getByPath(obj, "data.personalisedTagFeed.items.post");
		posts.forEach(post -> {
			Article article = new Article();
			article.setId((String) post.get("id"));
			article.setClapCount((int) post.get("clapCount"));
			article.setTitle((String) post.get("title"));
			article.setUniqueSlug((String) post.get("uniqueSlug"));
			log.info("article:{}", article);
			articles.add(article);
		});

		// 获取next中的source
		String nextSource = (String) JSONUtil.getByPath(obj, "data.personalisedTagFeed.pagingInfo.next.source");
		log.info("prevSource:{},nextSource:{}", source, nextSource);
		return nextSource;
	}
	
	@Deprecated
	int queryClapCountByPostId(String postId) {
		String reqBody = "[ {\"operationName\":\"ClapCountQuery\",\"variables\":{\"postId\":\"" + postId
				+ "\"},\"query\":\"query ClapCountQuery($postId: ID!) {\\n  postResult(id: $postId) {\\n    __typename\\n    ... on Post {\\n      id\\n      clapCount\\n      __typename\\n    }\\n  }\\n}\\n\"}]";
		String resBody = request(HttpMethod.POST,"https://medium.com/_/graphql", reqBody, parseHeaderFromFile.parseGraphQLHeaders());
		Pattern pattern = Pattern.compile("\"clapCount\":\\s*(\\d+)");
		Matcher matcher = pattern.matcher(resBody);
		if (matcher.find()) {
			String clapCount = matcher.group(1);
			log.info("postId:{}, clapCount:{}", postId, clapCount);
			return Integer.parseInt(clapCount);
		}
		return -1;
	}

	String request(HttpMethod method,String url, String reqBody, Map<String, String> headers) {
		HttpHeaders hs = new HttpHeaders();
		if(headers!=null) {
			headers.remove("Cookie");
			headers.put("Cookie", cookie);
			headers.forEach(hs::set);
		}
		log.info("请求url:{} {}", method, url);
		log.info("请求header:{}", headers);
		log.info("请求body:{}", reqBody);
		HttpEntity<String> entity = new HttpEntity<>(reqBody, hs);
		ResponseEntity<String> res = null;
		if(method==HttpMethod.POST) {
			res = restTemplate.postForEntity(url, entity, String.class);
		} else {
			res = restTemplate.getForEntity(url, String.class);
		}
		log.info("响应码:{}", res.getStatusCode());
		log.info("响应header:{}", res.getHeaders());
		if (res.getStatusCode().is2xxSuccessful()) {
			String body = res.getBody();
			log.info("响应body:{}", body);
			return body;
		}
		return null;
	}

}
